import urllib.request
from bs4 import BeautifulSoup
import json

#爬取动作小游戏的网页内容
def web_content(url):

    #请求网页
    req = urllib.request.Request(url)
    #print(req)
    #防止网站反爬，增加请求头信息
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36')
    #请求html内容
    html = urllib.request.urlopen(req).read().decode('gbk')

    #用BeautifulSoup库对html内容做下一步的数据清洗
    soup = BeautifulSoup(html, features='lxml')
    #print(soup)
    return soup

#爬取动作标签下面的子标签
def action_child_tag(soup):
    soup = soup.find_all('div',{'class': 'mi-lr'})
    for item in soup:
        key = item.find('a')
        #print(key.string)
        value = item.find_all('span')
        #定义一个字典
        dic = {}
        for i in value:
            href = i.find('a')
            # 获取a标签的文本值
            k = href.string
            #获取a标签的网址
            v = href['href']
            dic[k] = v
            #print(i.string)
        #print(dic)
        dictionary[key.string] = dic
    return dictionary

#主函数
if __name__ == '__main__':
    #定义请求地址
    url = 'http://www.4399.com/'
    #定义字典用于保存页面当中所有的大标题与子内容
    dictionary = {}
    # 定义程序当中可能用到的变量
    beautifulSoup = web_content(url)
    text = action_child_tag(beautifulSoup)
    # with open('dictionary.txt','w',encoding='utf-8') as file:
    #     file.write(json.dumps(text,ensure_ascii=False))
    # file.close()
